from pathlib import Path
import random
import time
import itertools
import re
import csv
import pandas

from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup

from browser import Browser
import config



class Parser(Browser):

    """ Этот класс представляет парсер """

    def __init__(self, email, password, display_browser):

        self.link = 'http://msk.galara.ru/'
        self.email = email
        self.password = password
        self.display_browser = display_browser
        self.browser = self.get_browser(self.display_browser)


    def wait(self, f=2, b=3.5):

        """ Чуть-чуть подождать """

        number = random.uniform(f, b)
        print(f'Ждем {number}\n')
        time.sleep(number)


    def wirte_csv_minimum_price(self, data):

        with open('data.csv', 'a') as file:

            writer = csv.writer(file)
            writer.writerow((
                data[0],
                data[1],
                data[2],
                data[3],
                data[4],
                data[5]
            ))


    def write_csv(self, data):

        """ Записываем данные в csv файл """

        with open('data.csv', 'a') as file:

            writer = csv.writer(file)
            writer.writerow((
                data[0],
                data[1],
                data[2],
                data[3],
                data[4]
            ))


    def convert_to_bs(self):

        """ Получаем объект BeautifulSoup """

        html_source = self.browser.page_source
        page = BeautifulSoup(html_source, 'lxml')

        return page


    def get_saved_page(self, file_name):

        """ Открываем сохраненную страницу для парсинга и возвращаем ее """

        folder = Path('saved_html/')
        page = folder/file_name

        with open(page, 'r', encoding='utf-8') as file:

            text = file.read()
            page = BeautifulSoup(text, 'lxml')

            return page


    def start_parser(self):

        """ Начать парсинг """

        def login():

            """ Авторизуемся на сайте """

            self.launch_browser(self.link)

            email = self.browser.find_element(
                By.XPATH,
                '//div[@class="left-column"]/div[1]//form//input[@id="maemail"]'
            )
            email.send_keys(self.email)

            self.wait(1, 1.5)

            password = self.browser.find_element(
                By.XPATH,
                '//div[@class="left-column"]/div[1]//form//input[@id="mapass"]'
            )
            password.send_keys(self.password)

            enter = self.browser.find_element(
                By.XPATH,
                '//div[@class="left-column"]/div[1]//form//input[@value="Войти"]'
            ).click()

            self.wait(2, 3)


        def get_brand_catalog():

            """ Получаем каталог брендов """

            self.launch_browser(self.link)
            page = self.convert_to_bs()

            brand_catalog = page.find('div', {'class': 'left-column'}) \
                .find('div', {'class': 'alpha-block'}) \
                .find_all('span')

            result = [
                [element.find('a').text, 'http://msk.galara.ru' + element.find('a')['href']]
                for element in brand_catalog
            ]

            return result


        def get_all_brands_letter(catalog):

            """ Получаем все бренды буквы """

            def add_products_to_brand(catalog, products):

                """ Добавляем все продукты в бренд """

                for brand, products in itertools.zip_longest(catalog, products):

                    brand.append(products)

                return catalog


            # ВЫШЕ ОПРЕДЕЛЕНИЕ ФУНКЦИЙ

            result = []

            for product in catalog:

                self.launch_browser(product[1])

                titles = self.browser.find_elements(
                    By.XPATH,
                    '//div[@class="main-column"]//table[@class="brands"]//div/a'
                )
                ready_titles = [item.text for item in titles]

                links = self.browser.find_elements(
                    By.XPATH,
                    '//div[@class="main-column"]//table[@class="brands"]//div//a'
                )
                ready_links = [
                    item.get_attribute('href')
                    for item in links
                ]

                done = []

                for (title, link) in itertools.zip_longest(ready_titles, ready_links):

                    data = [title, link]
                    done.append(data)

                result.append(done)
                self.wait(1.5, 3)


            result = add_products_to_brand(catalog, result)

            return result


        def get_second_product_name(products):

            """ Работаем со всеми продуктами """

            def go_to_brand(brands):

                """ Работаем с брендом """

                def get_all_products(link):

                    """ Получаем все продукты """

                    self.launch_browser(link)

                    a = self.browser.find_elements(
                        By.XPATH,
                        '//div[@id="list_brand_lines"]/div/div[@class="name"]//a'
                    )

                    titles = [item.text for item in a]
                    links = [item.get_attribute('href') for item in a]

                    return (titles, links)


                def join_lists(lists):

                    """ Объединяем списки """

                    result = []
                    titles, links = lists

                    for title, link in itertools.zip_longest(titles, links):

                        data = [title, link]
                        result.append(data)

                    return result


                def get_all_information(products):

                    """ Получаем всю информацию с продукта """

                    def get_title(product):

                        """ Получаем название продукта """

                        title = self.browser.find_element(
                            By.XPATH,
                            '//div[@class="main-column"]//h1'
                        ).text

                        return title


                    def get_first_group_parameters(product):

                        """ Получаем первую группу параметров """

                        result = []

                        parameters_one = self.browser.find_elements(
                            By.XPATH,
                            '//div[@class="product-char i-block"]/div[@class="name"]'
                        )
                        done_parameters_one = [item.text for item in parameters_one]

                        parameters_two = self.browser.find_elements(
                            By.XPATH,
                            '//div[@class="product-char i-block"]/div[@class="items"]'
                        )
                        done_parameters_two = [item.text for item in parameters_two]


                        for (p_one, p_two) in itertools.zip_longest(done_parameters_one, done_parameters_two):

                            data = [p_one, p_two]
                            result.append(data)

                        return result


                    def get_second_group_parameters(product):

                        """ Получаем вторую группу параметров """

                        def get_additionally():

                            """ Получаем дополнительную информацию о продукте """

                            done_additionally = []

                            page = self.convert_to_bs()
                            additionallys = page.find('table', {'class': 'list-price'}) \
                                .find_all('tr')[1:]

                            for item in additionallys:

                                item = item.contents[0]

                                for td in item:

                                    try:

                                        result = td.find('div').text

                                    except:

                                        result = ''

                                    done_additionally.append(result)

                            return done_additionally


                        # ВЫШЕ ОБЪЯВЛЕНИЯ ФУНКЦИЙ

                        result = []
                        done_volumes = []

                        volumes = self.browser.find_elements(
                            By.XPATH,
                            '//table[@class="list-price"]//td[2]'
                        )

                        for volume in volumes:

                            try:

                                done_volumes.append(volume.text)

                            except:

                                done_volumes.append('Объем не указан')


                        done_additionally = get_additionally()

                        price = self.browser.find_elements(
                            By.XPATH,
                            '//table[@class="list-price"]//td[3]'
                        )
                        done_price = [item.text for item in price]

                        title = self.browser.find_elements(
                            By.XPATH,
                            '//table[@class="list-price"]//td[1]'
                        )
                        done_title = [item.text for item in title]


                        for (volume, price, title, additionally) in itertools.zip_longest(done_volumes, done_price, done_title, done_additionally):

                            data = [volume, price, title, additionally]
                            result.append(data)

                        return result


                    def get_all_images(product):

                        """ Получаем все изображения """

                        main_image = self.browser.find_element(
                            By.XPATH,
                            '//div[@class="product-img i-block"]//div[@class="img data-big-photo"]//img'
                        ).get_attribute('src')
                        done_main_image = main_image

                        other_images = self.browser.find_elements(
                            By.XPATH,
                            '//div[@class="product-img i-block"]//table[@class="data-list"]//img'
                        )
                        ready_other_image = [item.get_attribute('src') for item in other_images]

                        ready_other_image.insert(0, done_main_image)

                        return ready_other_image


                    # ВЫШЕ ОБЪЯВЛЕНИЕ ФУНКЦИЙ

                    done = []

                    for product in products:

                        self.launch_browser(product[1])

                        first_group_parameters = get_first_group_parameters(product)
                        second_group_parameters = get_second_group_parameters(product)
                        all_images = get_all_images(product)
                        title = get_title(product)
                        data = (title, first_group_parameters, second_group_parameters, all_images)
                        done.append(data)

                        self.wait(2, 4)

                    return done


                done = []

                # ВЫШЕ ОБЪЯВЛЕНИЕ ФУНКЦИЙ

                for brand in brands:

                    title = brand[0]
                    link = brand[1]

                    print(f'Все продукты бренда {title}')

                    all_products = get_all_products(link)
                    combined_lists = join_lists(all_products)
                    all_information = get_all_information(combined_lists)

                    all_information.insert(0, title)
                    done.append(all_information)

                return done


            # ВЫШЕ ОБЪЯВЛЕНИЕ ФУНКЦИЙ

            done = []

            for letter in products:

                p = letter[2]
                d = go_to_brand(p)
                d.insert(0, p)
                done.append(d)

            return done


        def add_letter(letters, products):

            """ Добавляем букву """

            result = []

            for l, p in itertools.zip_logest(letters, products):

                data = [l, p]
                result.append(data)

            return result


        def sort_data(data):

            def add_lowest_price(description):

                """ Выявляем меньшую цену """

                if not ';' in description[1]: return

                prices = description[1].split(';')

                temp_prices = []

                for price in prices:

                    price = int(price.split(' ')[-1])
                    temp_prices.append(price)

                result = min(temp_prices)

                return result


            def get_description_one(product):

                """ Получаем описание продукта """

                result = []

                for item in product:

                    description = ''.join(item)
                    result.append(description)

                result = ', '.join(result)

                return result


            def get_description_two(product):

                """ Получаем параметры продукта """

                done = []
                result = []

                for item in product:

                    title = item[2].strip()
                    result.append(title)

                result = list(set(result))
                result = [[item] for item in result]

                for i in result:

                    for item in product:

                        if 'нет в наличии' in item[1]: continue

                        title = item[2].strip()
                        price = re.findall(r'\d', item[1])
                        price = ''.join(price)

                        if title == i[0]: i.append([item[3], item[0], price])


                for item in result:

                    if len(item) == 1: continue
                    done.append(item)


#                for item in done:
#
#                    title = item[0]
#                    desc = [i for i in item[1] if i]
#                    desc = ' - '.join(desc)


                return done


            def get_full_name_and_description(name, description):

                """ Получаем полное название и описание """

                done = []
                result = []

                for item in description:

                    title = item[0]
                    full_title = f'{name} {title}'
                    result.append([full_title, item[1:]])

                for item in result:

                    for j in item[1]:

                        done.append([item[0], ' - '.join(j)])

                return done


            def get_description(description):

                """ Получаем готовое описание """

                products = []

                for item in description:

                    products.append(item[0])

                products = list(set(products))
                products = [[item] for item in products]

                for product in products:

                    for item in description:

                        title = item[0]
                        #if item[1][1] == '-': desc = item[1][3:]
                        if product[0] == title: product.append(item[1:])

                return products


            # ВЫШЕ ОБЪЯВЛЕНИЕ ФУНКЦИЙ

            letter_l = []
            title_l = []
            description_one_l = []
            images_l = []
            description_l = []
            min_number_l = []

            for brand in data:

                for i in brand:

                    if type(i[0]) == list: continue
                    letter = i[0][0]
                    product_name_one = i[0]

                    for j in i:

                        if len(j[0]) == 1: continue
                        if 'нет в наличии' in j[2]: continue

                        print(j)

                        product_name_two = f'{i[0]} {j[0]}'
                        product_name_three = j[2][-1][-1]

                        description_one = get_description_one(j[1])
                        description_two = get_description_two(j[2])
                        description = get_full_name_and_description(product_name_two, description_two)
                        products = get_description(description)

                        bla = []

                        for product in products:

                            title = product[0]
                            description = [item[0] for item in product[1:]]
                            done = []
                            for p in description:
                                if p[1] == '-': done.append(p[3:])
                            description = '; '.join(done)
                            bla.append([title, description])

                        for p in bla:

                            title = p[0]
                            description = p[1]
                            images = j[3]
                            images = ', '.join(images)
                            min_number = add_lowest_price(p)

                            if min_number:

                                letter_l.append(letter)
                                title_l.append(title)
                                description_one_l.append(description_one)
                                images_l.append(images)
                                description_l.append(description)
                                min_number_l.append(min_number)
                                #data = (letter, title, description_one, images, description, min_number)
                                #self.wirte_csv_minimum_price(data)

                            else:

                                letter_l.append(letter)
                                title_l.append(title)
                                description_one_l.append(description_one)
                                images_l.append(images)
                                description_l.append(description)
                                min_number_l.append('')
                                #data = (letter, title, description_one, images, description)
                                #self.write_csv(data)

            df = pandas.DataFrame({'Буква': letter_l,
                                   'Название': title_l,
                                   'Характеристики': description_one_l,
                                   'Изображения': images_l,
                                   'Параметры': description_l,
                                   'Минимальная цена': min_number_l})
            df.to_excel('./data.xlsx')

        # ВЫШЕ ОПРЕДЕЛЕНИЕ ФУНКЦИЙ

        login()
        brand_catalog = get_brand_catalog()

        #brand_catalog.reverse()
        #brand_catalog = brand_catalog[:1]

        all_brands_letters = get_all_brands_letter(brand_catalog)
        second_product_name = get_second_product_name(all_brands_letters)
        sort_data(second_product_name)
